
import requests, re, html as Html
from lxml import etree
class Spider:

    #这个是url,会经常被和谐如果没有请联系我
    url = 'https://www.rewrewrwq.xyz/'

    def __init__(self):
        self.session = requests.session()
        self.headers = {
            'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 SocketLog(tabid=603&client_id=)',

        }
        pass

    def get_category(self):
        """
        数据1
        获取所有板块
        :return:
        """
        response = self.session.get(self.url,headers=self.headers)
        if response.status_code != 200:
            raise Exception('请求获取板块数据异常')
        text = response.text
        return self.__parse_cate(text)

    def __parse_cate(self,html_content):
        """
        板块数据处理
        :return:
        """
        html = etree.HTML(html_content)
        cates = html.xpath('//div[@class="bm bmw  flg cl"]')
        cates_count = len(cates) + 1
        base_xpath = '//div[@class="fl bm"]/div[{index}][@class="bm bmw  flg cl"]'
        pattern_today = '<em class="xw0 xi1" title="今日"> \((\d+)\)</em></dt>'
        pattern_theme1 = '<dd><em>主题: (\d+)</em>'
        pattern_theme2 = '<em>主题: <span title="(\d+)">'
        pattern_tie1 = '<em>帖数: <span title="(\d+)">'
        pattern_tie2 = '<em>帖数: (\d+)'
        pattern_last = '最后发表: ([\s\S]*?)</a>'
        pattern_cate = '<div id="category_(\d+)"'
        pattern_child_cate = 'forum-(\d+)-1.html'
        urls = []
        data = []
        for v in range(1,cates_count):
            i = str(v)
            l_div = html.xpath(base_xpath.format(index=i))
            div = l_div[0]
            child_div = etree.tostring(div, method='html').decode('utf-8').strip()
            s_html = etree.HTML(child_div)
            url_html = s_html.xpath('//dl')
            pid = v
            cate_name = s_html.xpath('//div[@class="bm_h cl"]//h2/span/text()')
            cate_id = re.findall(pattern_cate,child_div)
            data.append({
                    'name': cate_name[0],
                    'pid': 0,
                    'link': '',
                    'themes': 0,
                    'ties': 0,
                    'today': 0,
                    'last_date': 0,
                    'cate_id':cate_id[0] if cate_id else 0
                })

            for child in url_html:
                child_dl = etree.tostring(child, method='html').decode('utf-8').strip()
                child_dl = Html.unescape(child_dl)
                child_dl_html = etree.HTML(child_dl)
                url = child_dl_html.xpath('//dt/a/@href')
                urls.append(url)

                name = child_dl_html.xpath('//dt/a/text()')
                theme = re.findall(pattern_theme1,child_dl)
                if not theme:
                    theme = re.findall(pattern_theme2,child_dl)
                tie = re.findall(pattern_tie1,child_dl)
                if not tie:
                    tie = re.findall(pattern_tie2,child_dl)
                today_count = re.findall(pattern_today,child_dl)

                last_date = re.findall(pattern_last,child_dl)
                url = url[0] if url else ''
                if url:
                    cate_id = re.findall(pattern_child_cate,url)
                cate_info = {
                    'name': name[0] if name else '',
                    'pid': pid,
                    'link': url,
                    'themes': theme[0] if theme else 0,
                    'ties': tie[0] if tie else 0,
                    'today_count': today_count[0] if today_count else 0,
                    'last_date': last_date[0].strip() if last_date else '',
                    'cate_id':cate_id[0] if cate_id else 0
                }
                data.append(cate_info)
        return data


spder = Spider()

#获取论坛板块主题
category = spder.get_category()


print(category)